<!DOCTYPE html>
<html lang="en" itemscope itemtype="https://schema.org/WebPage">
  <head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <title>Vitess / Topology Service</title>

        <!-- Webfont -->
    <link href='http://fonts.googleapis.com/css?family=Roboto:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic' rel='stylesheet' type='text/css'>
    
    <!--<link rel="shortcut icon" type="image/png" href="/favicon.png">-->

    <!-- Bootstrap -->
    <link href="/libs/bootstrap/css/bootstrap.min.css" rel="stylesheet">

    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
    <!--[if lt IE 9]>
      <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
      <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->

    <!-- Styles -->
    <link rel="stylesheet" type="text/css" href="/css/site.css" />
    <!-- Font Awesome icons -->
    <link href="/libs/font-awesome-4.4.0/css/font-awesome.min.css"
          rel="stylesheet"
          type="text/css">
    
    <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
    <!-- Include all compiled plugins (below), or include individual files as needed -->
    <script src="/libs/bootstrap/js/bootstrap.min.js"></script>


    <!-- metadata -->
    <meta name="og:title" content="Vitess / Topology Service"/>
    <meta name="og:image" content="/images/vitess_logo_with_border.svg"/>
    <meta name="og:description" content="Vitess is a database clustering system for horizontal scaling of MySQL."/>

    <link rel="icon" href="/images/vitess_logo_icon_size.png" type="image/png">

    
    <script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
  ga('create', 'UA-60219601-1', 'auto');
  ga('send', 'pageview');
</script>

    
  </head>
  <body class="docs" id="top_of_page">

    <span id="toc-depth" data-toc-depth="h2,h3"></span>


    <nav id="common-nav" class="navbar navbar-fixed-top inverse">
  <div class="container">
    <!-- Brand and toggle get grouped for better mobile display -->
    <div class="navbar-header">
      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar-collapse-1">
        <span class="sr-only">Toggle navigation</span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
      <a class="navbar-brand" href="/">
        <img class="logo" src="/images/vitess_logo_with_border.svg" alt="Vitess">
      </a>
    </div>

    <!-- Collect the nav links, forms, and other content for toggling -->
    <div class="collapse navbar-collapse" id="navbar-collapse-1">
      <ul class="nav navbar-nav navbar-right" id="standard-menu-links">
        <li><a href="/overview/">Overview</a></li>
        <li><a href="/user-guide/introduction.html">Guides</a></li>
        <li><a href="/reference/vitess-api.html">Reference</a></li>
        <li><a href="http://blog.vitess.io">Blog</a></li>
        <li><a href="https://github.com/youtube/vitess"><i class="fa fa-github"></i> GitHub</a></li>
        <!-- Hide link to blog unless we have actual posts -->
        <!-- <li><a href="/blog/" title="">Blog</a></li> -->
      </ul>
      <ul class="nav nav-stacked mobile-left-nav-menu" id="collapsed-left-menu">
                <li class="submenu">
          <h4 class="arrow-r no-top-margin">Overview</h4>
          <ul style="display: none">
            <li><a href="/overview/">What is Vitess</a></li>
            <li><a href="/overview/scaling-mysql.html">Scaling MySQL with Vitess</a></li>
            <li><a href="/overview/concepts.html">Key Concepts</a></li>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Getting Started</h4>
          <ul style="display: none">
            <li style="padding-bottom: 0"><a href="/getting-started/">Run Vitess on Kubernetes</a>
              <ul style="display: block">
                <li style="padding-bottom: 0"><a href="/getting-started/docker-build.html">Custom Docker Build</a></li>
              </ul>
            </li>
            <li><a href="/getting-started/local-instance.html">Run Vitess Locally</a></li>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">User Guide</h4>
          <ul style="display: none">
            <li><a href="/user-guide/introduction.html">Introduction</a>
            <li><a href="/user-guide/client-libraries.html">Client Libraries</a>
            <li><a href="/user-guide/backup-and-restore.html">Backing Up Data</a>
            <li><a href="/user-guide/reparenting.html">Reparenting</a></li>
            <li style="padding-bottom: 0"><a href="/user-guide/schema-management.html">Schema Management</a></li>
              <ul style="display: block">
                <li style="padding-bottom: 0"><a href="/user-guide/vschema.html">VSchema Guide</a></li>
                <li style="padding-bottom: 0"><a href="/user-guide/schema-swap.html">Schema Swap (Tutorial)</a></li>
              </ul>
            <li style="padding-bottom: 0"><a href="/user-guide/sharding.html">Sharding</a>
              <ul style="display: block">
                <li><a href="/user-guide/horizontal-sharding-workflow.html">Horizontal Sharding (Tutorial, automated)</a></li>
                <li><a href="/user-guide/horizontal-sharding.html">Horizontal Sharding (Tutorial, manual)</a></li>
                <li><a href="/user-guide/sharding-kubernetes-workflow.html">Sharding in Kubernetes (Tutorial, automated)</a></li>
                <li style="padding-bottom: 0"><a href="/user-guide/sharding-kubernetes.html">Sharding in Kubernetes (Tutorial, manual)</a></li>
              </ul>
            </li>
            <li><a href="/user-guide/vitess-sequences.html">Vitess Sequences</a></li>
            <li><a href="/user-guide/mysql-server-protocol.html">MySQL Server Protocol</a></li>
            <li><a href="/user-guide/vitess-replication.html">Vitess and Replication</a></li>
            <li><a href="/user-guide/update-stream.html">Update Stream</a></li>
            <li><a href="/user-guide/row-based-replication.html">Row Based Replication</a></li>
            <li><a href="/user-guide/topology-service.html">Topology Service</a></li>
            <li><a href="/user-guide/transport-security-model.html">Transport Security Model</a></li>
            <li style="padding-bottom: 0"><a href="/user-guide/launching.html">Launching</a>
              <ul style="display: block">
                <li><a href="/user-guide/scalability-philosophy.html">Scalability Philosophy</a></li>
                <li><a href="/user-guide/production-planning.html">Production Planning</a></li>
                <li><a href="/user-guide/server-configuration.html">Server Configuration</a></li>
                <li><a href="/user-guide/twopc.html">2PC Guide</a></li>
                <li style="padding-bottom: 0"><a href="/user-guide/troubleshooting.html">Troubleshooting</a></li>
              </ul>
            </li>
            <li><a href="/user-guide/upgrading.html">Upgrading</a></li>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Reference Guides</h4>
          <ul style="display: none">
            <li><a href="/reference/vitess-api.html">Vitess API</a>
            <li><a href="/reference/vtctl.html">vtctl Commands</a>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Other Resources</h4>
          <ul style="display: none">
            <li><a href="/resources/presentations.html">Presentations</a>
            <li><a href="http://blog.vitess.io/">Blog</a>
            <li><a href="/resources/roadmap.html">Roadmap</a>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Contributing</h4>
          <ul style="display: none">
            <li><a href="/contributing/">Contributing to Vitess</a>
            <li><a href="/contributing/github-workflow.html">GitHub Workflow</a>
            <li><a href="/contributing/code-reviews.html">Code Reviews</a>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Internal</h4>
          <ul style="display: none">
            <li><a href="/internal/">Overview</a>
            <li><a href="/internal/release-instructions.html">Release Instructions</a>
            <li><a href="/internal/publish-website.html">Publish Website</a>
          </ul>
        </li>

        <div>
          <form method="get" action="/search/">
            <input id="search-form" name="q" type="text" placeholder="Search">
          </form>
        </div>

        <li><a href="https://github.com/youtube/vitess" id="collapsed-left-menu-repo-link"><i class="fa fa-github"></i> GitHub</a></li>
      </ul>
    </div><!-- /.navbar-collapse -->
  </div><!-- /.container-fluid -->
</nav>

    
<div id="masthead">
  <div class="container">
    <div class="row">
      <div class="col-md-9">
        <h1>Topology Service</h1>
      </div>
    </div>
  </div>
</div>


<div class="container">
    <div class="row scroll-margin" id="toc-content-row">
    <div class="col-md-2" id="leftCol">
      <ul class="nav nav-stacked mobile-left-nav-menu" id="sidebar">
                <li class="submenu">
          <h4 class="arrow-r no-top-margin">Overview</h4>
          <ul style="display: none">
            <li><a href="/overview/">What is Vitess</a></li>
            <li><a href="/overview/scaling-mysql.html">Scaling MySQL with Vitess</a></li>
            <li><a href="/overview/concepts.html">Key Concepts</a></li>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Getting Started</h4>
          <ul style="display: none">
            <li style="padding-bottom: 0"><a href="/getting-started/">Run Vitess on Kubernetes</a>
              <ul style="display: block">
                <li style="padding-bottom: 0"><a href="/getting-started/docker-build.html">Custom Docker Build</a></li>
              </ul>
            </li>
            <li><a href="/getting-started/local-instance.html">Run Vitess Locally</a></li>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">User Guide</h4>
          <ul style="display: none">
            <li><a href="/user-guide/introduction.html">Introduction</a>
            <li><a href="/user-guide/client-libraries.html">Client Libraries</a>
            <li><a href="/user-guide/backup-and-restore.html">Backing Up Data</a>
            <li><a href="/user-guide/reparenting.html">Reparenting</a></li>
            <li style="padding-bottom: 0"><a href="/user-guide/schema-management.html">Schema Management</a></li>
              <ul style="display: block">
                <li style="padding-bottom: 0"><a href="/user-guide/vschema.html">VSchema Guide</a></li>
                <li style="padding-bottom: 0"><a href="/user-guide/schema-swap.html">Schema Swap (Tutorial)</a></li>
              </ul>
            <li style="padding-bottom: 0"><a href="/user-guide/sharding.html">Sharding</a>
              <ul style="display: block">
                <li><a href="/user-guide/horizontal-sharding-workflow.html">Horizontal Sharding (Tutorial, automated)</a></li>
                <li><a href="/user-guide/horizontal-sharding.html">Horizontal Sharding (Tutorial, manual)</a></li>
                <li><a href="/user-guide/sharding-kubernetes-workflow.html">Sharding in Kubernetes (Tutorial, automated)</a></li>
                <li style="padding-bottom: 0"><a href="/user-guide/sharding-kubernetes.html">Sharding in Kubernetes (Tutorial, manual)</a></li>
              </ul>
            </li>
            <li><a href="/user-guide/vitess-sequences.html">Vitess Sequences</a></li>
            <li><a href="/user-guide/mysql-server-protocol.html">MySQL Server Protocol</a></li>
            <li><a href="/user-guide/vitess-replication.html">Vitess and Replication</a></li>
            <li><a href="/user-guide/update-stream.html">Update Stream</a></li>
            <li><a href="/user-guide/row-based-replication.html">Row Based Replication</a></li>
            <li><a href="/user-guide/topology-service.html">Topology Service</a></li>
            <li><a href="/user-guide/transport-security-model.html">Transport Security Model</a></li>
            <li style="padding-bottom: 0"><a href="/user-guide/launching.html">Launching</a>
              <ul style="display: block">
                <li><a href="/user-guide/scalability-philosophy.html">Scalability Philosophy</a></li>
                <li><a href="/user-guide/production-planning.html">Production Planning</a></li>
                <li><a href="/user-guide/server-configuration.html">Server Configuration</a></li>
                <li><a href="/user-guide/twopc.html">2PC Guide</a></li>
                <li style="padding-bottom: 0"><a href="/user-guide/troubleshooting.html">Troubleshooting</a></li>
              </ul>
            </li>
            <li><a href="/user-guide/upgrading.html">Upgrading</a></li>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Reference Guides</h4>
          <ul style="display: none">
            <li><a href="/reference/vitess-api.html">Vitess API</a>
            <li><a href="/reference/vtctl.html">vtctl Commands</a>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Other Resources</h4>
          <ul style="display: none">
            <li><a href="/resources/presentations.html">Presentations</a>
            <li><a href="http://blog.vitess.io/">Blog</a>
            <li><a href="/resources/roadmap.html">Roadmap</a>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Contributing</h4>
          <ul style="display: none">
            <li><a href="/contributing/">Contributing to Vitess</a>
            <li><a href="/contributing/github-workflow.html">GitHub Workflow</a>
            <li><a href="/contributing/code-reviews.html">Code Reviews</a>
          </ul>
        </li>
        <li class="submenu">
          <h4 class="arrow-r">Internal</h4>
          <ul style="display: none">
            <li><a href="/internal/">Overview</a>
            <li><a href="/internal/release-instructions.html">Release Instructions</a>
            <li><a href="/internal/publish-website.html">Publish Website</a>
          </ul>
        </li>

        <div>
          <form method="get" action="/search/">
            <input id="search-form" name="q" type="text" placeholder="Search">
          </form>
        </div>

      </ul>
    </div>
    <div class="col-md-3" id="rightCol">
      <div class="nav nav-stacked" id="tocSidebar">
        <div id="toc"></div>
      </div>
    </div>
    <div class="col-md-7" id="centerCol">
      <div id="centerTextCol">
        <h1 id="topology-service">Topology Service</h1>

<p>This document describes the Topology Service, a key part of the Vitess
architecture. This service is exposed to all Vitess processes, and is used to
store small pieces of configuration data about the Vitess cluster, and provide
cluster-wide locks. It also supports watches, which we will use soon.</p>

<p>Concretely, the Topology Service features are implemented by
a <a href="http://en.wikipedia.org/wiki/Distributed_lock_manager">Lock Server</a>, referred
to as Topology Server in the rest of this document. We use a plug-in
implementation and we support multiple Lock Servers (Zookeeper, etcd, Consul, …)
as backends for the service.</p>

<h2 id="requirements-and-usage">Requirements and Usage</h2>

<p>The Topology Service is used to store information about the Keyspaces, the
Shards, the Tablets, the Replication Graph, and the Serving Graph. We store
small data structures (a few hundred bytes) per object.</p>

<p>The main contract for the Topology Server is to be very highly available and
consistent. It is understood it will come at a higher latency cost and very low
throughput.</p>

<p>We never use the Topology Server as an RPC mechanism, nor as a storage system
for logs. We never depend on the Topology Server being responsive and fast to
serve every query.</p>

<p>The Topology Server must also support a Watch interface, to signal when certain
conditions occur on a node. This is used for instance to know when keyspaces
topology changes (for resharding for instance).</p>

<h3 id="global-vs-local">Global vs Local</h3>

<p>We differentiate two instances of the Topology Server: the Global instance, and
the per-cell Local instance:</p>

<ul>
<li>The Global instance is used to store global data about the topology that
doesn’t change very often, for instance information about Keyspaces and
Shards. The data is independent of individual instances and cells, and needs
to survive a cell going down entirely.</li>
<li>There is one Local instance per cell, that contains cell-specific information,
and also rolled-up data from the global + local cell to make it easier for
clients to find the data. The Vitess local processes should not use the Global
topology instance, but instead the rolled-up data in the Local topology
server as much as possible.</li>
</ul>

<p>The Global instance can go down for a while and not impact the local cells (an
exception to that is if a reparent needs to be processed, it might not work). If
a Local instance goes down, it only affects the local tablets in that instance
(and then the cell is usually in bad shape, and should not be used).</p>

<p>Furthermore, the Vitess processes will not use the Global nor the Local Topology
Server to serve individual queries. They only use the Topology Server to get the
topology information at startup and in the background, but never to directly
serve queries.</p>

<h3 id="recovery">Recovery</h3>

<p>If a local Topology Server dies and is not recoverable, it can be wiped out. All
the tablets in that cell then need to be restarted so they re-initialize their
topology records (but they won’t lose any MySQL data).</p>

<p>If the global Topology Server dies and is not recoverable, this is more of a
problem. All the Keyspace / Shard objects have to be re-created. Then the cells
should recover.</p>

<h2 id="global-data">Global Data</h2>

<p>This section describes the data structures stored in the global instance of the
topology server.</p>

<h3 id="keyspace">Keyspace</h3>

<p>The Keyspace object contains various information, mostly about sharding: how is
this Keyspace sharded, what is the name of the sharding key column, is this
Keyspace serving data yet, how to split incoming queries, …</p>

<p>An entire Keyspace can be locked. We use this during resharding for instance,
when we change which Shard is serving what inside a Keyspace. That way we
guarantee only one operation changes the keyspace data concurrently.</p>

<h3 id="shard">Shard</h3>

<p>A Shard contains a subset of the data for a Keyspace. The Shard record in the
global topology contains:</p>

<ul>
<li>the MySQL Master tablet alias for this shard</li>
<li>the sharding key range covered by this Shard inside the Keyspace</li>
<li>the tablet types this Shard is serving (master, replica, batch, …), per cell
if necessary.</li>
<li>if during filtered replication, the source shards this shard is replicating
from</li>
<li>the list of cells that have tablets in this shard</li>
<li>shard-global tablet controls, like blacklisted tables no tablet should serve
in this shard</li>
</ul>

<p>A Shard can be locked. We use this during operations that affect either the
Shard record, or multiple tablets within a Shard (like reparenting), so multiple
jobs don’t concurrently alter the data.</p>

<h3 id="vschema-data">VSchema Data</h3>

<p>The VSchema data contains sharding and routing information for
the <a href="https://github.com/youtube/vitess/blob/master/doc/VTGateV3Features.md">VTGate V3</a> API.</p>

<h2 id="local-data">Local Data</h2>

<p>This section describes the data structures stored in the local instance (per
cell) of the topology server.</p>

<h3 id="tablets">Tablets</h3>

<p>The Tablet record has a lot of information about a single vttablet process
running inside a tablet (along with the MySQL process):</p>

<ul>
<li>the Tablet Alias (cell+unique id) that uniquely identifies the Tablet</li>
<li>the Hostname, IP address and port map of the Tablet</li>
<li>the current Tablet type (master, replica, batch, spare, …)</li>
<li>which Keyspace / Shard the tablet is part of</li>
<li>the sharding Key Range served by this Tablet</li>
<li>user-specified tag map (to store per installation data for instance)</li>
</ul>

<p>A Tablet record is created before a tablet can be running (either by <code class="prettyprint">vtctl
InitTablet</code> or by passing the <code class="prettyprint">init_*</code> parameters to vttablet). The only way a
Tablet record will be updated is one of:</p>

<ul>
<li>The vttablet process itself owns the record while it is running, and can
change it.</li>
<li>At init time, before the tablet starts</li>
<li>After shutdown, when the tablet gets deleted.</li>
<li>If a tablet becomes unresponsive, it may be forced to spare to make it
unhealthy when it restarts.</li>
</ul>

<h3 id="replication-graph">Replication Graph</h3>

<p>The Replication Graph allows us to find Tablets in a given Cell / Keyspace /
Shard. It used to contain information about which Tablet is replicating from
which other Tablet, but that was too complicated to maintain. Now it is just a
list of Tablets.</p>

<h3 id="serving-graph">Serving Graph</h3>

<p>The Serving Graph is what the clients use to find the per-cell topology of a
Keyspace. It is a roll-up of global data (Keyspace + Shard). vtgates only open a
small number of these objects and get all they need quickly.</p>

<h4 id="srvkeyspace">SrvKeyspace</h4>

<p>It is the local representation of a Keyspace. It contains information on what
shard to use for getting to the data (but not information about each individual
shard):</p>

<ul>
<li>the partitions map is keyed by the tablet type (master, replica, batch, …) and
the values are list of shards to use for serving.</li>
<li>it also contains the global Keyspace fields, copied for fast access.</li>
</ul>

<p>It can be rebuilt by running <code class="prettyprint">vtctl RebuildKeyspaceGraph</code>. It is
automatically rebuilt when a tablet starts up in a cell and the SrvKeyspace
for that cell / keyspace doesn&#39;t exist yet. It will also be changed
during horizontal and vertical splits.</p>

<h4 id="srvvschema">SrvVSchema</h4>

<p>It is the local roll-up for the VSchema. It contains the VSchema for all
keyspaces in a single object.</p>

<p>It can be rebuilt by running <code class="prettyprint">vtctl RebuildVSchemaGraph</code>. It is automatically
rebuilt when using <code class="prettyprint">vtctl ApplyVSchema</code> (unless prevented by flags).</p>

<h2 id="workflows-involving-the-topology-server">Workflows Involving the Topology Server</h2>

<p>The Topology Server is involved in many Vitess workflows.</p>

<p>When a Tablet is initialized, we create the Tablet record, and add the Tablet to
the Replication Graph. If it is the master for a Shard, we update the global
Shard record as well.</p>

<p>Administration tools need to find the tablets for a given Keyspace / Shard:
first we get the list of Cells that have Tablets for the Shard (global topology
Shard record has these) then we use the Replication Graph for that Cell /
Keyspace / Shard to find all the tablets then we can read each tablet record.</p>

<p>When a Shard is reparented, we need to update the global Shard record with the
new master alias.</p>

<p>Finding a tablet to serve the data is done in two stages: vtgate maintains a
health check connection to all possible tablets, and they report which keyspace
/ shard / tablet type they serve. vtgate also reads the SrvKeyspace object, to
find out the shard map. With these two pieces of information, vtgate can route
the query to the right vttablet.</p>

<p>During resharding events, we also change the topology a lot. An horizontal split
will change the global Shard records, and the local SrvKeyspace records. A
vertical split will change the global Keyspace records, and the local
SrvKeyspace records.</p>

<h2 id="implementations">Implementations</h2>

<p>The Topology Server interface is defined in our code in <code class="prettyprint">go/vt/topo/server.go</code>
and we also have a set of unit tests for it in <code class="prettyprint">go/vt/topo/test</code>.</p>

<p>This part describes the two implementations we have, and their specific
behavior.</p>

<p>If starting from scratch, please use the <code class="prettyprint">zk2</code>, <code class="prettyprint">etcd2</code> or <code class="prettyprint">consul</code>
implementations, as we are deprecating the old <code class="prettyprint">zookeeper</code> and <code class="prettyprint">etcd</code>
implementations. See the migration section below if you want to migrate.</p>

<h3 id="zookeeper-zk2-implementation-new-version-of-zookeeper">Zookeeper <code class="prettyprint">zk2</code> Implementation (new version of <code class="prettyprint">zookeeper</code>)</h3>

<p>This is the recommended implementation when using Zookeeper. The old <code class="prettyprint">zookeeper</code>
implementation is deprecated, see next section.</p>

<p>The global cell typically has around 5 servers, distributed one in each
cell. The local cells typically have 3 or 5 servers, in different server racks /
sub-networks for higher resilience. For our integration tests, we use a single
ZK server that serves both global and local cells.</p>

<p>We provide the <code class="prettyprint">zk</code> utility for easy access to the topology data in
Zookeeper. It can list, read and write files inside any Zoopeeker server. Just
specify the <code class="prettyprint">-server</code> parameter to point to the Zookeeper servers. Note the
vtctld UI can also be used to see the contents of the topology data.</p>

<p>To configure a Zookeeper installation, let&#39;s start with the global cell
service. It is described by the addresses of the servers (comma separated list),
and by the root directory to put the Vitess data in. For instance, assuming we
want to use servers <code class="prettyprint">global_server1,global_server2</code> in path <code class="prettyprint">/vitess/global</code>:</p>
<div class="highlight"><pre><code class="language-sh" data-lang="sh"><span class="c"># First create the directory in the global server:</span>
zk -server global_server1,global_server2 touch -p /vitess/global

<span class="c"># Set the following flags to let Vitess use this global server:</span>
<span class="c"># -topo_implementation zk2</span>
<span class="c"># -topo_global_server_address global_server1,global_server2</span>
<span class="c"># -topo_global_root /vitess/global</span>
</code></pre></div>
<p>Then to add a cell whose local topology servers <code class="prettyprint">cell1_server1,cell1_server2</code>
will store their data under the directory <code class="prettyprint">/vitess/cell1</code>:</p>
<div class="highlight"><pre><code class="language-sh" data-lang="sh"><span class="nv">TOPOLOGY</span><span class="o">=</span><span class="s2">&quot;-topo_implementation zk2 -topo_global_server_address global_server1,global_server2 -topo_global_root /vitess/global&quot;</span>

<span class="c"># Reference cell1 in the global topology service:</span>
vtctl <span class="nv">$TOPOLOGY</span> AddCellInfo <span class="se">\</span>
  -server_address cell1_server1,cell1_server2 <span class="se">\</span>
  -root /vitess/cell1 <span class="se">\</span>
  cell1
</code></pre></div>
<p>If only one cell is used, the same Zookeeper instance can be used for both
global and local data. A local cell record still needs to be created, just use
the same server address, and very importantly a <em>different</em> root directory.</p>

<h4 id="implementation-details">Implementation Details</h4>

<p>We use the following paths:</p>

<p><em>Global Cell</em>:</p>

<ul>
<li>Election path: <code class="prettyprint">elections/&lt;name&gt;</code></li>
<li>CellInfo path: <code class="prettyprint">cells/&lt;cell name&gt;/CellInfo</code></li>
<li>Keyspace: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/Keyspace</code></li>
<li>Shard: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/shards/&lt;shard&gt;/Shard</code></li>
<li>VSchema: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/VSchema</code></li>
</ul>

<p><em>Local Cell</em>:</p>

<ul>
<li>Tablet: <code class="prettyprint">tablets/&lt;cell&gt;-&lt;uid&gt;/Tablet</code></li>
<li>Replication Graph: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/shards/&lt;shard&gt;/ShardReplication</code></li>
<li>SrvKeyspace: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/SrvKeyspace</code></li>
<li>SrvVSchema: <code class="prettyprint">SvrVSchema</code></li>
</ul>

<p>For locks, we create a subdirectory called <code class="prettyprint">locks</code> under either the keyspace
directory or the shard directory.</p>

<p>Both locks and master election are implemented using ephemeral, sequential files
which are stored in their respective directory.</p>

<p>We store the proto3 binary data for each object. The <code class="prettyprint">zk</code> utility can decode
these files when using the <code class="prettyprint">-p</code> option of the <code class="prettyprint">cat</code> command:</p>
<div class="highlight"><pre><code class="language-sh" data-lang="sh"><span class="nv">$ </span>zk --server localhost:15014 cat -p /global/keyspaces/test_keyspace/shards/-80/Shard
master_alias: &lt;
  cell: <span class="s2">&quot;test_nj&quot;</span>
  uid: 62344
&gt;
key_range: &lt;
  end: <span class="s2">&quot;\200&quot;</span>
&gt;
served_types: &lt;
  tablet_type: MASTER
&gt;
served_types: &lt;
  tablet_type: REPLICA
&gt;
served_types: &lt;
  tablet_type: RDONLY
&gt;
cells: <span class="s2">&quot;test_nj&quot;</span>
</code></pre></div>
<h3 id="old-zookeeper-zookeeper-implementaion-deprecated-use-zk2-instead">Old Zookeeper <code class="prettyprint">zookeeper</code> Implementaion (deprecated, use <code class="prettyprint">zk2</code> instead)</h3>

<p>This old <code class="prettyprint">zookeeper</code> topology service is deprecated, and will be removed in
Vitess version 2.2. Please use <code class="prettyprint">zk2</code> instead, and see the <code class="prettyprint">topo2topo</code> section
below for migration.</p>

<p>Our Zookeeper implementation is based on a configuration file that describes
where the global and each local cell ZK instances are. When adding a cell, all
processes that may access that cell should be restarted with the new
configuration file.</p>

<p>The global cell typically has around 5 servers, distributed one in each
cell. The local cells typically have 3 or 5 servers, in different server racks /
sub-networks for higher resilience. For our integration tests, we use a single
ZK server that serves both global and local cells.</p>

<p>We sometimes store both data and sub-directories in a path (for a keyspace for
instance). We use JSON to encode the data.</p>

<p>For locking, we use an auto-incrementing file name in the <code class="prettyprint">/action</code> subdirectory
of the object directory. We also move them to <code class="prettyprint">/actionlogs</code> when the lock is
released. And we have a purge process to clear the old locks (which should be
run on a crontab, typically).</p>

<p>Note the paths used to store global and per-cell data do not overlap, so a
single ZK can be used for both global and local ZKs. This is however not
recommended, for reliability reasons.</p>

<ul>
<li>Keyspace: <code class="prettyprint">/zk/global/vt/keyspaces/&lt;keyspace&gt;</code></li>
<li>Shard: <code class="prettyprint">/zk/global/vt/keyspaces/&lt;keyspace&gt;/shards/&lt;shard&gt;</code></li>
<li>Tablet: <code class="prettyprint">/zk/&lt;cell&gt;/vt/tablets/&lt;uid&gt;</code></li>
<li>Replication Graph: <code class="prettyprint">/zk/&lt;cell&gt;/vt/replication/&lt;keyspace&gt;/&lt;shard&gt;</code></li>
<li>SrvKeyspace: <code class="prettyprint">/zk/&lt;cell&gt;/vt/ns/&lt;keyspace&gt;</code></li>
<li>SrvVSchema: <code class="prettyprint">/zk/&lt;cell&gt;/vt/vschema</code></li>
</ul>

<p>We provide the &#39;zk&#39; utility for easy access to the topology data in
Zookeeper. For instance:</p>
<div class="highlight"><pre><code class="language-sh" data-lang="sh"><span class="c"># NOTE: We do not set the ZK_CLIENT_CONFIG environment variable here,</span>
<span class="c"># as the zk tool connects to a specific server. </span>
<span class="nv">$ </span>zk -server &lt;server address&gt; ls /zk/global/vt/keyspaces/user
action
actionlog
shards
</code></pre></div>
<h3 id="etcd-etcd2-implementation-new-version-of-etcd">etcd <code class="prettyprint">etcd2</code> Implementation (new version of <code class="prettyprint">etcd</code>)</h3>

<p>This topology service plugin is meant to use etcd clusters as storage backend
for the topology data. This topology service supports version 3 and up of the
etcd server.</p>

<p>This implementation is named <code class="prettyprint">etcd2</code> because it supersedes our previous
implementation <code class="prettyprint">etcd</code>. Note that the storage format has been changed with the
<code class="prettyprint">etcd2</code> implementation, i.e. existing data created by the previous <code class="prettyprint">etcd</code>
implementation must be migrated manually (See migration section below).</p>

<p>To configure an <code class="prettyprint">etcd2</code> installation, let&#39;s start with the global cell
service. It is described by the addresses of the servers (comma separated list),
and by the root directory to put the Vitess data in. For instance, assuming we
want to use servers <code class="prettyprint">http://global_server1,http://global_server2</code> in path
<code class="prettyprint">/vitess/global</code>:</p>
<div class="highlight"><pre><code class="language-sh" data-lang="sh"><span class="c"># Set the following flags to let Vitess use this global server,</span>
<span class="c"># and simplify the example below:</span>
<span class="c"># -topo_implementation etcd2</span>
<span class="c"># -topo_global_server_address http://global_server1,http://global_server2</span>
<span class="c"># -topo_global_root /vitess/global</span>
<span class="nv">TOPOLOGY</span><span class="o">=</span><span class="err">&quot;</span>-topo_implementation etcd2 -topo_global_server_address http://global_server1,http://global_server2 -topo_global_root /vitess/global
</code></pre></div>
<p>Then to add a cell whose local topology servers
<code class="prettyprint">http://cell1_server1,http://cell1_server2</code> will store their data under the
directory <code class="prettyprint">/vitess/cell1</code>:</p>
<div class="highlight"><pre><code class="language-sh" data-lang="sh"><span class="c"># Reference cell1 in the global topology service:</span>
<span class="c"># (the TOPOLOGY variable is defined in the previous section)</span>
vtctl <span class="nv">$TOPOLOGY</span> AddCellInfo <span class="se">\</span>
  -server_address http://cell1_server1,http://cell1_server2 <span class="se">\</span>
  -root /vitess/cell1 <span class="se">\</span>
  cell1
</code></pre></div>
<p>If only one cell is used, the same etcd instances can be used for both
global and local data. A local cell record still needs to be created, just use
the same server address and, very importantly, a <em>different</em> root directory.</p>

<h4 id="implementation-details">Implementation Details</h4>

<p>We use the following paths:</p>

<p><em>Global Cell</em>:</p>

<ul>
<li>Election path: <code class="prettyprint">elections/&lt;name&gt;</code></li>
<li>CellInfo path: <code class="prettyprint">cells/&lt;cell name&gt;/CellInfo</code></li>
<li>Keyspace: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/Keyspace</code></li>
<li>Shard: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/shards/&lt;shard&gt;/Shard</code></li>
<li>VSchema: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/VSchema</code></li>
</ul>

<p><em>Local Cell</em>:</p>

<ul>
<li>Tablet: <code class="prettyprint">tablets/&lt;cell&gt;-&lt;uid&gt;/Tablet</code></li>
<li>Replication Graph: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/shards/&lt;shard&gt;/ShardReplication</code></li>
<li>SrvKeyspace: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/SrvKeyspace</code></li>
<li>SrvVSchema: <code class="prettyprint">SvrVSchema</code></li>
</ul>

<p>For locks, we use a subdirectory named <code class="prettyprint">locks</code> in the directory to lock, and an
ephemeral file in that subdirectory (it is associated with a lease, whose TTL
can be set with the <code class="prettyprint">-topo_etcd_lease_duration</code> flag, defaults to 30
seconds). The ephemeral file with the lowest ModRevision has the lock, the
others wait for files with older ModRevisions to disappear.</p>

<p>Master elections also use a subdirectory, named after the election Name, and use
a similar method as the locks, with ephemeral files.</p>

<p>We store the proto3 binary data for each object (as the v3 API allows us to store binary data).</p>

<h3 id="old-etcd-etcd-implementaion-deprecated-use-etcd2-instead">Old etcd <code class="prettyprint">etcd</code> Implementaion (deprecated, use <code class="prettyprint">etcd2</code> instead)</h3>

<p>This old <code class="prettyprint">etcd</code> topology service is deprecated, and will be removed in
Vitess version 2.2. Please use <code class="prettyprint">etcd2</code> instead, and see the <code class="prettyprint">topo2topo</code> section
below for migration.</p>

<p>Our etcd implementation is based on a command-line parameter that gives the
location(s) of the global etcd server. Then we query the path <code class="prettyprint">/vt/cells</code> and
each file in there is named after a cell, and contains the list of etcd servers
for that cell. Each cell server files are stored in <code class="prettyprint">/vt/</code>.</p>

<p>We use the <code class="prettyprint">_Data</code> filename to store the data, JSON encoded.</p>

<p>For locking, we store a <code class="prettyprint">_Lock</code> file with various contents in the directory that
contains the object to lock.</p>

<p>We use the following paths:</p>

<ul>
<li>Keyspace: <code class="prettyprint">/vt/keyspaces/&lt;keyspace&gt;/_Data</code></li>
<li>Shard: <code class="prettyprint">/vt/keyspaces/&lt;keyspace&gt;/&lt;shard&gt;/_Data</code></li>
<li>Tablet: <code class="prettyprint">/vt/tablets/&lt;cell&gt;-&lt;uid&gt;/_Data</code></li>
<li>Replication Graph: <code class="prettyprint">/vt/replication/&lt;keyspace&gt;/&lt;shard&gt;/_Data</code></li>
<li>SrvKeyspace: <code class="prettyprint">/vt/ns/&lt;keyspace&gt;/_Data</code></li>
<li>SrvVSchema: <code class="prettyprint">/vt/ns/_VSchema</code></li>
</ul>

<h3 id="consul-consul-implementation">Consul <code class="prettyprint">consul</code> Implementation</h3>

<p>This topology service plugin is meant to use Consul clusters as storage backend
for the topology data.</p>

<p>To configure a <code class="prettyprint">consul</code> installation, let&#39;s start with the global cell
service. It is described by the address of a server,
and by the root node path to put the Vitess data in (it cannot start with <code class="prettyprint">/</code>). For instance, assuming we
want to use servers <code class="prettyprint">global_server:global_port</code> with node path
<code class="prettyprint">vitess/global</code>:</p>
<div class="highlight"><pre><code class="language-sh" data-lang="sh"><span class="c"># Set the following flags to let Vitess use this global server,</span>
<span class="c"># and simplify the example below:</span>
<span class="c"># -topo_implementation consul</span>
<span class="c"># -topo_global_server_address global_server:global_port</span>
<span class="c"># -topo_global_root vitess/global</span>
<span class="nv">TOPOLOGY</span><span class="o">=</span><span class="err">&quot;</span>-topo_implementation consul -topo_global_server_address global_server:global_port -topo_global_root vitess/global
</code></pre></div>
<p>Then to add a cell whose local topology server
<code class="prettyprint">cell1_server1:cell1_port</code> will store their data under the
directory <code class="prettyprint">vitess/cell1</code>:</p>
<div class="highlight"><pre><code class="language-sh" data-lang="sh"><span class="c"># Reference cell1 in the global topology service:</span>
<span class="c"># (the TOPOLOGY variable is defined in the previous section)</span>
vtctl <span class="nv">$TOPOLOGY</span> AddCellInfo <span class="se">\</span>
  -server_address cell1_server1:cell1_port <span class="se">\</span>
  -root vitess/cell1 <span class="se">\</span>
  cell1
</code></pre></div>
<p>If only one cell is used, the same consul instances can be used for both
global and local data. A local cell record still needs to be created, just use
the same server address and, very importantly, a <em>different</em> root node path.</p>

<h4 id="implementation-details">Implementation Details</h4>

<p>We use the following paths:</p>

<p><em>Global Cell</em>:</p>

<ul>
<li>Election path: <code class="prettyprint">elections/&lt;name&gt;</code></li>
<li>CellInfo path: <code class="prettyprint">cells/&lt;cell name&gt;/CellInfo</code></li>
<li>Keyspace: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/Keyspace</code></li>
<li>Shard: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/shards/&lt;shard&gt;/Shard</code></li>
<li>VSchema: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/VSchema</code></li>
</ul>

<p><em>Local Cell</em>:</p>

<ul>
<li>Tablet: <code class="prettyprint">tablets/&lt;cell&gt;-&lt;uid&gt;/Tablet</code></li>
<li>Replication Graph: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/shards/&lt;shard&gt;/ShardReplication</code></li>
<li>SrvKeyspace: <code class="prettyprint">keyspaces/&lt;keyspace&gt;/SrvKeyspace</code></li>
<li>SrvVSchema: <code class="prettyprint">SvrVSchema</code></li>
</ul>

<p>For locks, we use a file named <code class="prettyprint">Lock</code> in the directory to lock, and the regular
Consul Lock API.</p>

<p>Master elections use a single lock file (the Election path) and the regular
Consul Lock API. The contents of the lock file is the ID of the current master.</p>

<p>Watches use the Consul long polling Get call. They cannot be interrupted, so we
use a long poll whose duration is set by the <code class="prettyprint">-topo_consul_watch_poll_duration</code>
flag. Canceling a watch may have to wait until the end of a polling cycle with
that duration before returning.</p>

<p>We store the proto3 binary data for each object.</p>

<h2 id="running-in-only-one-cell">Running In Only One Cell</h2>

<p>The topology service is meant to be distributed across multiple cells, and
survive single cell outages. However, one common usage is to run a Vitess
cluster in only one cell / region. This part explains how to do this, and later
on upgrade to multiple cells / regions.</p>

<p>If running in a single cell, the same topology service can be used for both
global and local data.  A local cell record still needs to be created, just use
the same server address and, very importantly, a <em>different</em> root node path.</p>

<p>In that case, just running 3 servers for topology service quorum is probably
sufficient. For instance, 3 etcd servers. And use their address for the local
cell as well. Let&#39;s use a short cell name, like <code class="prettyprint">local</code>, as the local data in
that topology server will later on be moved to a different topology service,
which will have the real cell name.</p>

<h3 id="extending-to-more-cells">Extending To More Cells</h3>

<p>To then run in multiple cells, the current topology service needs to be split
into a global instance and one local instance per cell. Whereas, the initial
setup had 3 topology servers (used for global and local data), we recommend to
run 5 global servers across all cells (for global topology data) and 3 local
servers per cell (for per-cell topology data).</p>

<p>To migrate to such a setup, start by adding the 3 local servers in the second
cell and run <code class="prettyprint">vtctl AddCellinfo</code> as was done for the first cell. Tablets and
vtgates can now be started in the second cell, and used normally.</p>

<p>vtgate can then be configured with a list of cells to watch for tablets using
the <code class="prettyprint">-cells_to_watch</code> command line parameter. It can then use all tablets in all
cells to route traffic. Note this is necessary to access the master in another
cell.</p>

<p>After the extension to two cells, the original topo service contains both the
global topology data, and the first cell topology data. The more symetrical
configuration we&#39;re after would be to split that original service into two: a
global one that only contains the global data (spread across both cells), and a
local one to the original cells. To achieve that split:</p>

<ul>
<li>Start up a new local topology service in that original cell (3 more local
servers in that cell).</li>
<li>Pick a name for that cell, different from <code class="prettyprint">local</code>.</li>
<li>Use <code class="prettyprint">vtctl AddCellInfo</code> to configure it.</li>
<li>Make sure all vtgates can see that new local cell (again, using
<code class="prettyprint">-cells_to_watch</code>).</li>
<li>Restart all vttablets to be in that new cell, instead of the <code class="prettyprint">local</code> cell name
used before.</li>
<li>Use <code class="prettyprint">vtctl RemoveKeyspaceCell</code> to remove all mentions of the <code class="prettyprint">local</code> cell in
all keyspaces.</li>
<li>Use <code class="prettyprint">vtctl RemoveCellInfo</code> to remove the global configurations for that
<code class="prettyprint">local</code> cell.</li>
<li>Remove all remaining data in the global topology service that are in the old
local server root.</li>
</ul>

<p>After this split, the configuration is completely symetrical:</p>

<ul>
<li>a global topology service, with servers in all cells. Only contains global
topology data about Keyspaces, Shards and VSchema. Typically it has 5 servers
across all cells.</li>
<li>a local topology service to each cell, with servers only in that cell. Only
contains local topology data about Tablets, and roll-ups of global data for
efficient access. Typically, it has 3 servers in each cell.</li>
</ul>

<h2 id="migration-between-implementations">Migration Between Implementations</h2>

<p>We provide the <code class="prettyprint">topo2topo</code> binary file to migrate between one implementation
and another of the topology service.</p>

<p>The process to follow in that case is:</p>

<ul>
<li>Start from a stable topology, where no resharding or reparenting is on-going.</li>
<li>Configure the new topology service so it has at least all the cells of the
source topology service. Make sure it is running.</li>
<li>Run the <code class="prettyprint">topo2topo</code> program with the right flags. <code class="prettyprint">-from_implementation</code>,
<code class="prettyprint">-from_root</code>, <code class="prettyprint">-from_server</code> describe the source (old) topology
service. <code class="prettyprint">-to_implementation</code>, <code class="prettyprint">-to_root</code>, <code class="prettyprint">-to_server</code> describe the
destination (new) topology service.</li>
<li>Run <code class="prettyprint">vtctl RebuildKeyspaceGraph</code> for each keyspace using the new topology
service flags.</li>
<li>Run <code class="prettyprint">vtctl RebuildVSchemaGraph</code> using the new topology service flags.</li>
<li>Restart all <code class="prettyprint">vtgate</code> using the new topology service flags. They will see the
same keyspaces / shards / tablets / vschema as before, as the topology was
copied over.</li>
<li>Restart all <code class="prettyprint">vttablet</code> using the new topology service flags. They may use the
same ports or not, but they will update the new topology when they start up,
and be visible from vtgate.</li>
<li>Restart all <code class="prettyprint">vtctld</code> processes using the new topology service flags. So that
the UI also shows the new data.</li>
</ul>

<p>Sample commands to migrate from deprecated <code class="prettyprint">zookeeper</code> to <code class="prettyprint">zk2</code>
topology would be:</p>
<div class="highlight"><pre><code class="language-sh" data-lang="sh"><span class="c"># Let&#39;s assume the zookeeper client config file is already</span>
<span class="c"># exported in $ZK_CLIENT_CONFIG, and it contains a global record</span>
<span class="c"># pointing to: global_server1,global_server2</span>
<span class="c"># an a local cell cell1 pointing to cell1_server1,cell1_server2</span>
<span class="c">#</span>
<span class="c"># The existing directories created by Vitess are:</span>
<span class="c"># /zk/global/vt/...</span>
<span class="c"># /zk/cell1/vt/...</span>
<span class="c">#</span>
<span class="c"># The new zk2 implementation can use any root, so we will use:</span>
<span class="c"># /vitess/global in the global topology service, and:</span>
<span class="c"># /vitess/cell1 in the local topology service.</span>

<span class="c"># Create the new topology service roots in global and local cell.</span>
zk -server global_server1,global_server2 touch -p /vitess/global
zk -server cell1_server1,cell1_server2 touch -p /vitess/cell1

<span class="c"># Store the flags in a shell variable to simplify the example below.</span>
<span class="nv">TOPOLOGY</span><span class="o">=</span><span class="s2">&quot;-topo_implementation zk2 -topo_global_server_address global_server1,global_server2 -topo_global_root /vitess/global&quot;</span>

<span class="c"># Reference cell1 in the global topology service:</span>
vtctl <span class="nv">$TOPOLOGY</span> AddCellInfo <span class="se">\</span>
  -server_address cell1_server1,cell1_server2 <span class="se">\</span>
  -root /vitess/cell1 <span class="se">\</span>
  cell1

<span class="c"># Now copy the topology. Note the old zookeeper implementation doesn&#39;t need</span>
<span class="c"># any server or root parameter, as it reads ZK_CLIENT_CONFIG.</span>
topo2topo <span class="se">\</span>
  -from_implementation zookeeper <span class="se">\</span>
  -to_implementation zk2 <span class="se">\</span>
  -to_server global_server1,global_server2 <span class="se">\</span>
  -to_root /vitess/global <span class="se">\</span>

<span class="c"># Rebuild SvrKeyspace objects in new service, for each keyspace.</span>
vtctl <span class="nv">$TOPOLOGY</span> RebuildKeyspaceGraph keyspace1
vtctl <span class="nv">$TOPOLOGY</span> RebuildKeyspaceGraph keyspace2

<span class="c"># Rebuild SrvVSchema objects in new service.</span>
vtctl <span class="nv">$TOPOLOGY</span> RebuildVSchemaGraph

<span class="c"># Now restart all vtgate, vttablet, vtctld processes replacing:</span>
<span class="c"># -topo_implementation zookeeper</span>
<span class="c"># With:</span>
<span class="c"># -topo_implementation zk2</span>
<span class="c"># -topo_global_server_address global_server1,global_server2</span>
<span class="c"># -topo_global_root /vitess/global</span>
<span class="c">#</span>
<span class="c"># After this, the ZK_CLIENT_CONF file and environment variables are not needed</span>
<span class="c"># any more.</span>
</code></pre></div>
      </div>
    </div>
  </div>

</div>

    <div class="page-spacer"></div>
    <footer role="contentinfo" id="site-footer">
  <nav role="navigation" class="menu bottom-menu">
    
    <a href="https://groups.google.com/forum/#!forum/vitess" target="_blank">Contact: vitess@googlegroups.com</a>&nbsp;&nbsp;<b>·</b>&nbsp;&nbsp;
    <a href="https://groups.google.com/forum/#!forum/vitess-announce" target="_blank">Announcements</a>&nbsp;&nbsp;<b>·</b>&nbsp;&nbsp;
    &#169; 2017 <a href="/">Vitess</a> powered by <a href="https://developers.google.com/open-source/">Google Inc</a>
  </nav>
</footer>


        <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
<!--    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>   -->
    <!-- Include all compiled plugins (below), or include individual files as needed -->
<!--
    <script src="/libs/bootstrap/js/bootstrap.min.js"></script>
-->
    <!-- Include the common site javascript -->
    <script src="/js/common.js" type="text/javascript" charset="utf-8"></script>


  </body>
</html>
